package com.guiji.quartz.domain;

import cn.hutool.core.date.DateUtil;
import cn.hutool.crypto.SecureUtil;
import cn.hutool.http.HttpException;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONUtil;
import cn.wanghaomiao.xpath.exception.NoSuchAxisException;
import cn.wanghaomiao.xpath.exception.NoSuchFunctionException;
import cn.wanghaomiao.xpath.exception.XpathSyntaxErrorException;
import cn.wanghaomiao.xpath.model.JXDocument;
import com.guiji.common.utils.spring.SpringUtils;
import com.guiji.quartz.formatter.DataTemplateFormatter;
import com.guiji.quartz.service.ISpiderNewsService;
import com.guiji.quartz.service.RedisService;
import com.guiji.quartz.task.DataSaveUtil;
import jodd.util.StringUtil;
import lombok.SneakyThrows;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @program: cms-vue-plus
 * @description:
 * @author: gaoX
 * @create: 2021-08-25 15:44
 **/
public class SinaTotal {
	private static final Logger log = LoggerFactory.getLogger(SinaTotal.class);

	public void mainMethod() throws NoSuchFunctionException, XpathSyntaxErrorException, NoSuchAxisException {
		List<String> entry = getEntry();
		HttpResponse response = HttpRequest.get(entry.get(0))
			.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
			.header("Accept-Encoding", "gzip, deflate")
			.setConnectionTimeout(15000)
			.execute();
		String resp = response.body();
		List<String> jsUrl = getJsUrl(resp);
		for(String jsurl:jsUrl){
			String resp1 = null;
			try {
				HttpResponse response1 = HttpRequest.get(jsurl)
					.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
					.header("Accept-Encoding", "gzip, deflate")
					.setConnectionTimeout(15000)
					.execute();
				resp1 = response1.body();
			} catch (HttpException e) {
				continue;
			}
			List<String> lastUrl = getLastUrl(resp1);
			//爬取具体内容
			getContent(lastUrl);

		}
	}

	private static void getContent(List<String> listContent) throws NoSuchFunctionException, XpathSyntaxErrorException, NoSuchAxisException {
		for(String url:listContent){
			System.out.println("???????????????????????"+url+"??????????????????????");
			HttpResponse response = HttpRequest.get(url)
				.setConnectionTimeout(15000)
				.execute();
			String doc = response.body();
			JXDocument jxDocument = new JXDocument(doc);
			String title = null;
			try {
				title = (String) jxDocument.sel("//h1/text()").get(0);
				if(StringUtil.isAllEmpty(title)){
					title = (String) jxDocument.sel("//h1/text()").get(1);
				}
			} catch (Exception e) {
				System.out.println("标题错误-------------"+url+"-------------------标题错误");
				continue;
			}
			StringBuffer content = new StringBuffer();

			try {
				jxDocument.sel("//div[@class='article']//p/text()").forEach(s->{
					content.append(s+"$$$");
				});
				if(content.length()<5){
					jxDocument.sel("//div[@class='article']//p/span/text()").forEach(s->{
						content.append(s+"$$$");
					});
				}
			} catch (Exception e) {
				System.out.println("获取内容失败"+url);
				continue;
			}

			List<Object> imgs = jxDocument.sel("//div[@class='img_wrapper']//img/@src");
			List<String> newImgs = dealPic(imgs);

			Document document = Jsoup.parse(doc);
			DataTemplateFormatter dataTemplateFormatter = new DataTemplateFormatter();

			Date date = null;
			try {
				date = dataTemplateFormatter.format(document.getElementsByClass("date").toString().replaceAll("<.*?>",""));
			} catch (Exception e) {
				SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
				String s = sdf.format(new Date());
				date = dataTemplateFormatter.format(s+" 00:00  ");
			}

			StringBuffer categorySb = new StringBuffer();

			try {
				jxDocument.sel("//div[@class='channel-path']//a/text()").forEach(s->{
					categorySb.append(s);
				});
			} catch (Exception e) {
				System.out.println("获取目录失败"+url);
				continue;
			}
			String category = categorySb.toString();
			if(category.contains("经")||category.contains("券")||category.contains("股")||category.contains("私募")){
				category="新浪财经总榜";
			}else if(category.contains("军事")){
				category="新浪军事总榜";
			}else if(category.contains("社会")){
				category="新浪社会总榜";
			}else if(category.contains("科技")){
				category="新浪科技总榜";
			}else{
				continue;
			}


			if (content.length() > 5000 || content.toString().replaceAll("\\$|[\\s\\p{Zs}]","").length() < 20 || newImgs.size() > 20) {
				continue;
			}
			try {
				DataSaveUtil.saveData(title, content.toString(), newImgs, url, date, "https://ent.sina.com.cn/", category);
				log.info(Thread.currentThread().getName() + "-----------------sinatotal----------------");
			} catch (Exception e) {
				continue;
			}
		}
	}

	private static List<String> dealPic(List<Object> imgList){
		List<String> newImg = new ArrayList<>();
		for (Object s : imgList) {
			if (!s.toString().contains("http")) {
				String url = "https:" + s;
				try {
					String s1 = HttpUtil.get(url,15000);
					newImg.add(url);
				} catch (Exception e) {
					continue;
				}
			} else {
				try {
					String s1 = HttpUtil.get(s.toString());
					newImg.add(s.toString());
				} catch (Exception e) {
					continue;
				}
			}
		}
		return newImg;
	}

	private static List<String> getJsUrl(String content) {
		List<String> res = new ArrayList<>();
		String pattern = "//top.*data";
		Pattern p = Pattern.compile(pattern);
		Matcher m = p.matcher(content);
		while (m.find()) {
			res.add("http:"+m.group());
		}
		return res;
	}

	private static List<String> getLastUrl(String content) {
		List<String> res = new ArrayList<>();
		String pattern = "\"url\".*?html";
		Pattern p = Pattern.compile(pattern);
		Matcher m = p.matcher(content);
		while (m.find()) {
			res.add(m.group().replaceAll("\\\\","").replaceAll("\"url\":\"", ""));
		}
		return res;
	}


	public static List<String> getEntry() {
		List<String> listEntry = new ArrayList<>();
		listEntry.add("http://news.sina.com.cn/hotnews/");
		return listEntry;
	}

//	public static void main(String[] args) throws NoSuchFunctionException, XpathSyntaxErrorException, NoSuchAxisException {
//		String url = "https://news.sina.com.cn/c/2021-08-27/doc-ikqcfncc5393855.shtml";
//		HttpResponse response = HttpRequest.get(url)
//			.execute();
//		String doc = response.body();
//		JXDocument jxDocument = new JXDocument(doc);
//		String title = null;
//		try {
//			title = (String) jxDocument.sel("//h1/text()").get(0);
//			if(StringUtil.isAllEmpty(title)){
//				title = (String) jxDocument.sel("//h1/text()").get(1);
//			}
//		} catch (Exception e) {
//			System.out.println("标题错误-------------"+url+"-------------------标题错误");
//			return;
//		}
//		StringBuffer content = new StringBuffer();
//
//		try {
//			jxDocument.sel("//div[@class='article']//p/text()").forEach(s->{
//				content.append(s);
//			});
//			if(content.length()<5){
//				jxDocument.sel("//div[@class='article']//p/span/text()").forEach(s->{
//					content.append(s);
//				});
//			}
//		} catch (Exception e) {
//			return;
//		}
//
//		List<Object> imgs = jxDocument.sel("//div[@class='img_wrapper']//img/@src");
//		List<String> newImgs = dealPic(imgs);
//
//		Document document = Jsoup.parse(doc);
//		DataTemplateFormatter dataTemplateFormatter = new DataTemplateFormatter();
//
//		Date date = null;
//		try {
//			date = dataTemplateFormatter.format(document.getElementsByClass("date").toString().replaceAll("<.*?>",""));
//		} catch (Exception e) {
//			date = dataTemplateFormatter.format(s+" 00:00  ");
//		}
//
//		StringBuffer categorySb = new StringBuffer();
//
//		try {
//			jxDocument.sel("//div[@class='channel-path']//a/text()").forEach(s->{
//				categorySb.append(s);
//			});
//		} catch (Exception e) {
//			return;
//		}
//		System.out.println(categorySb);
//	}

}
